from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.spiders import CrawlSpider
from scrapy.http import FormRequest, Request
from scrapy import log
import scrapy


class DmozSpider(CrawlSpider):
    name = 'dmoz'
	allowed_domains = ["www.jyeoo.com"]
	start_urls = ["http://www.jyeoo.com"]
	
	urls = [
				"http://www.jyeoo.com/math/ques/search?f=0&s=0&t=0&q=8d1aaadf-4eec-4c53-8b17-48dfc8288f4a",
				"http://www.jyeoo.com/math/ques/search?f=0&s=0&t=0&q=475c9e72-7144-4774-ba97-cfb32414ecc7",
				] 	


	def parse(self, response):			
		print "hi1"				
		for i, url in enumerate(self.urls):     
			# notice: find out the VIPed url, if exsited, pass that to the next level through the meta attribute of request
			# judge the VIPed url exsited or not
			myrequest = Request(  	url,     #this is the urls extracted from former steps
									meta= { 'cookiejar' : i},
									# this is the cookies i figured out from the browser, no idea whether this could work or not
									cookies = { 'jyean':'uoAt257EP79CjLsbAtbC4wYf6rNDPpboJJS9ee-79P7mhDw8zJMkWq8lSFpsK4tUXErBPiQQkiYfSTSk6qCy04o425cLbX1HBm4-WUQ8F5sjBQoNPN0Z3m0WY6qElZec0',
												'pgv_pvi': '7626663936',
												'jye_math_ques_f' : '0',
												'jye_math_ques_s' : '0',
												'jye_math_ques_t' : '0',
												'jye_math_ques_0_q' : 'c84d9760-4abc-4de1-89d6-0fde547cf002~03a22d4f-2412-4651-8bb0-d8c94a0ab673~',
												'pgv_si' : 's1543265280',
												'LF_Email' : 'lh5712@163.com',
												'jyeoo': '840F477C78BFDB6D970F09E423EA2E50CB5C3857C56F84072EC89A17A5E5AB77A41A65FE7A0073168A69BCD585091FA656D883234DD32B2FB0B4B6CFB8EE05D1009D064694CD709607DE3DBBCCE45BDB595CB8254F594536268BC2976598C5C96A15B3289184FCF52EA0FBD36B35A115AC4F64723434D0B92BF8546DCD4654B17EB5645D360BBCA037A525F72646F8E6',
												'CNZZDATA2018550' : 'cnzz_eid%3D1688479147-1411128225-%26ntime%3D1413350434', 
												'JYERN' : '0.019439665833488107',},											
									callback = self.parse2)
			print myrequest
			return myrequest

	def parse2(self, response):    
		print "hi2"		
		myrequest = Request(	url = "http://www.jyeoo.com/math/ques/detail/cb265667-5648-480c-ab94-ffcbe35648ad",    # the url i used in this place is an VIPed url, normally this should be passed from the former level
								meta={'cookiejar': response.meta['cookiejar']},
								callback=self.parse3)
		print myrequest
		return myrequest
		
	def parse3(self, response):	
		print "hi3"
#		do some parse_work
		